package com.xu.crawler.processor;

import com.xu.crawler.dao.HistoryRecordDao;
import com.xu.crawler.dao.entity.HistoryRecordDO;
import com.xu.crawler.dao.entity.NewsDO;
import com.xu.crawler.util.SpringUtils;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.enums.SourceEnum;
import com.xu.crawler.pipeline.MongoPipeline;
import com.xu.crawler.enums.URLMapperEnum;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.utils.UrlUtils;

import javax.annotation.Resource;
import java.text.ParseException;
import java.util.List;
import java.util.Locale;

/**
 * Created by clear on 2018/7/11.
 */
@Service
public class ScmpPageProcessor implements PageProcessor {

    private Logger log = LoggerFactory.getLogger(ScmpPageProcessor.class);
    private Site site = Site.me()
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36")
            .setDomain(URLMapperEnum.SOUTH_CHINA_HOST.toString())
            .addHeader("cookie", "__cfduid=d44195256a4fdca824f3379c79f6add431531200378; Drupal.visitor.edition=O1; _ga=GA1.2.534851384.1531200382; cna=6IRqETIzyQ4CAXQYYyVzb3BR; _ain_uid=1531200634982.976411879.954515; SSESS335dc0b2361f3f85b12569cdeea87a16=0V65_kurzIw2xX4ezNVfSzLn2wE0GLWXQiOyQq9f_ko; SESS335dc0b2361f3f85b12569cdeea87a16=a03hpAvEhoCudDS_biEYdnevRMePO6TU4vQ0aDy_rUA; _ig=48f24114-eeab-4572-babc-1d570feff980; isg=BI6OXbGOWbH98e2Zq8RymRfL32Jb_lrnwRYRLrjX7RFHGy51IJ-iGTSSVwHSA0oh; acw_tc=AQAAAMA2ZhUq+g0A6o9qFy4T2TDw0G77")
            .addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
//            .addHeader("accept-encoding","gzip, deflate, br")
            .addHeader("accept-language","zh-CN,zh;q=0.9,en;q=0.8")
            .addHeader("pragma","no-cache")
            .addHeader("cache-control","no-cache")
            .addHeader(":authority","no-cache")
            .addHeader(":method","GET")
            .addHeader(":path","/sitemap_news.xml")
            .addHeader(":scheme","https")
            .addHeader("upgrade-insecure-requests","1")
            .addHeader("Referrer Policy","no-referrer-when-downgrade")
            .setRetryTimes(3).setSleepTime(1000);

    @Resource
    private HistoryRecordDao historyRecordDao;

    @Override
    public void process(Page page) {
        try {
            Html html = page.getHtml();
            List<String> urls = html.xpath("//loc/text()").all();
            /**
             * 获取昨日更新的最新一条记录,
             * 查询历史记录表，每次插入的时候记录最新的一条，按插入时间排序。
             */
            if (CollectionUtils.isNotEmpty(urls)) {
                //爬虫获取的最新的新闻url
                String newsUrl = urls.get(0);
                //历史记录中最新的新闻url
                if (historyRecordDao == null) {
                    historyRecordDao = SpringUtils.getBean(HistoryRecordDao.class);
                }
                String category = UrlUtils.getCategory(newsUrl, URLMapperEnum.SOUTH_CHINA_HOST.toString());
                String latestHistoryNewsUrl = historyRecordDao.queryLatestNewsUrl(SourceEnum.SCMP.getSourceId(), category);
                if (StringUtils.isBlank(latestHistoryNewsUrl)) {
                    //获取到今日需要插入的数据，添加到队列中
                    page.addTargetRequests(urls);
                } else {
                    for (String url : urls) {
                        if (url.equals(latestHistoryNewsUrl)) {
                            urls.clear();
                            break;
                        }
                        page.addTargetRequest(url);
                    }
                }
                //插入最新的新闻纪录
                saveHistoryRecord(newsUrl, category);
            }

            String title = html.xpath("//h1[@itemprop='name headline'][@id='page-title']/text()").toString();
            if (StringUtils.isBlank(title)) {
                page.getResultItems().setSkip(true);
            } else {
                NewsDO newsDO = renderNewsDO(html, title);
                page.setBaseDO(newsDO);
            }
        } catch (Throwable e) {
            log.error("process fail {}", e);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    private void saveHistoryRecord(String newsUrl, String category) {
        HistoryRecordDO historyRecordDO = new HistoryRecordDO();
        historyRecordDO.setUrl(newsUrl);
        historyRecordDO.setCreateTime(System.currentTimeMillis());
        historyRecordDO.setCategory(category);
        historyRecordDO.setSourceId(SourceEnum.SCMP.getSourceId());
        historyRecordDao.saveHistoryRecord(historyRecordDO);
    }

    private NewsDO renderNewsDO(Html html, String title) throws ParseException {
        NewsDO newsDO = new NewsDO();
        String jsonDate = html.xpath("//div[@itemprop='dateCreated'][@class='node-published']/@datetime").toString();
        newsDO.setPublishTime(StringUtils.isBlank(jsonDate) ? 0L : DateUtils.parseDate(jsonDate.substring(0, jsonDate.indexOf("+")), Locale.CHINA, "yyyy-MM-dd'T'HH:mm:ss").getTime());
        newsDO.setTitle(title);
        newsDO.setAuthor(html.xpath("//div[@class='scmp-v2-author-name']/allText()").toString());
        newsDO.setContent(html.xpath("//div[@class='col-xs-9 col-lg-9 pfcng-col pfcng-col-2']/div[@class='panel-pane pane-entity-field pane-node-body pane-first pos-0']/allText()").toString());
        String url = html.xpath("//link[@rel='canonical']/@href").toString();
        newsDO.setUrl(url);
        newsDO.setCategory(UrlUtils.getCategory(url, URLMapperEnum.SOUTH_CHINA_HOST.toString()));
        newsDO.setSourceId(SourceEnum.SCMP.getSourceId());
        newsDO.setHost(URLMapperEnum.SOUTH_CHINA_HOST.toString().substring(8));//截取https://
        return newsDO;
    }

    public static void main(String[] args) throws Exception {

        Spider spider = Spider.create(new ScmpPageProcessor());
        spider.addPipeline(new MongoPipeline());
        spider.addUrl("https://www.scmp.com/sitemap_news.xml").run();
//        spider.addUrl("https://www.scmp.com/news/hong-kong/politics/article/2131043/hong-kong-democracy-activist-edward-yiu-cleared-run-legco").thread(5).run();
    }
}
